const puppeteer = require('puppeteer');
const fs = require('fs/promises');
const { createLogger } = require('@modelcontextprotocol/sdk');

// 配置参数
const config = {
  targetUrl: 'https://www.baidu.com',
  maxRetries: 3,
  selectors: {
    container: '#hotsearch-content-wrapper',  // 更稳定的容器选择器
    title: '.c-single-text-ellipsis'          // 新版百度热点标题选择器
  },
  timeouts: {
    navigation: 45000,
    element: 20000
  }
};

const logger = createLogger('BaiduCrawler');

(async () => {
  let retryCount = 0;
  let browser;

  while (retryCount < config.maxRetries) {
    try {
      browser = await puppeteer.launch({
        headless: "new",
        args: ['--no-sandbox', '--disable-setuid-sandbox']
      });
      
      const page = await browser.newPage();
      await page.setViewport({ width: 1920, height: 1080 });

      // 智能等待策略
      await page.goto(config.targetUrl, {
        waitUntil: 'domcontentloaded',
        timeout: config.timeouts.navigation
      });

      // 滚动加载更多内容
      await autoScroll(page);

      // 等待内容加载
      const container = await page.waitForSelector(config.selectors.container, {
        timeout: config.timeouts.element,
        visible: true
      });

      if (!container) throw new Error('新闻容器未找到');

      // 动态获取热点数据
      const newsData = await page.evaluate((selector) => {
        return Array.from(document.querySelectorAll(selector)).map(el => ({
          title: el.textContent.trim(),
          url: el.closest('a')?.href || null
        }));
      }, config.selectors.title);

      // 增强数据格式
      const result = {
        timestamp: new Date().toISOString(),
        source: 'baidu',
        data: newsData
      };

      // 异步写入文件
      await fs.writeFile('baidu_news.json', JSON.stringify(result, null, 2));
      logger.info(`成功抓取${newsData.length}条新闻`);

      // 使用MCP工具记录截图
      await use_mcp_tool('browser-tools-mcp', 'takeScreenshot', {
        name: 'baidu-screenshot',
        fullPage: true
      });

      break;

    } catch (error) {
      retryCount++;
      logger.error(`抓取失败 (尝试 ${retryCount}/${config.maxRetries}): ${error.message}`);

      if (retryCount >= config.maxRetries) {
        logger.error('达到最大重试次数，终止任务');
        throw error;
      }

      // 指数退避重试
      await new Promise(resolve => 
        setTimeout(resolve, 3000 * Math.pow(2, retryCount)));
    } finally {
      if (browser) await browser.close();
    }
  }

  async function autoScroll(page) {
    await page.evaluate(async () => {
      await new Promise((resolve) => {
        let totalHeight = 0;
        const distance = 500;
        const timer = setInterval(() => {
          const scrollHeight = document.body.scrollHeight;
          window.scrollBy(0, distance);
          totalHeight += distance;
          if (totalHeight >= scrollHeight) {
            clearInterval(timer);
            resolve();
          }
        }, 500);
      });
    });
  }
})();
